{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python_defaultSpec_1595751830412",
   "display_name": "Python 3.7.4 64-bit ('tensorflow': conda)"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据集加载\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "# MNIST 0-10 手写数字数据集\n",
    "(x,y),(x_test,y_test) = keras.datasets.mnist.load_data()  # 加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "((60000, 28, 28), (60000,))"
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "x.shape,y.shape  # 训练图集 x 6W张28*28大小, y 表示0-9的标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(0, 255, 33.318421449829934)"
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "x.min(),x.max(),x.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "((10000, 28, 28), (10000,))"
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "x_test.shape,y_test.shape  # 测试集 1W张"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([5, 0, 4, 1], dtype=uint8)"
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "y[:4]  # 查看前四个标签"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "<tf.Tensor: shape=(4, 10), dtype=float32, numpy=\narray([[0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],\n       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],\n       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32)>"
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "y_onehot = tf.one_hot(y,depth=10)  # 转为向量,划分为10类\n",
    "y_onehot[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "((50000, 32, 32, 3), (50000, 1), (10000, 32, 32, 3), (10000, 1))"
     },
     "metadata": {},
     "execution_count": 7
    }
   ],
   "source": [
    "# CIFAR10/100 数据集  (5W个32*32三通道图片训练集,1W32*32测试集)\n",
    "(x,y),(x_test,y_test) = keras.datasets.cifar10.load_data()\n",
    "x.shape,y.shape,x_test.shape,y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(0, 255)"
     },
     "metadata": {},
     "execution_count": 8
    }
   ],
   "source": [
    "x.min(),x.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "array([[6],\n       [9],\n       [9],\n       [4]], dtype=uint8)"
     },
     "metadata": {},
     "execution_count": 9
    }
   ],
   "source": [
    "y[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numpy->Tensor  比起用for循环更方便,而且使用多线程并用Batch\n",
    "db = tf.data.Dataset.from_tensor_slices(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "TensorShape([32, 32, 3])"
     },
     "metadata": {},
     "execution_count": 12
    }
   ],
   "source": [
    "next(iter(db)).shape  # 迭代器 获得每张图片(Tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(TensorShape([32, 32, 3]), TensorShape([1]))"
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "# 返回两个样本\n",
    "db = tf.data.Dataset.from_tensor_slices((x_test,y_test))\n",
    "next(iter(db))[0].shape,next(iter(db))[1].shape  # 0:x ，1:y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle打散顺序\n",
    "db = tf.data.Dataset.from_tensor_slices((x_test,y_test))\n",
    "db = db.shuffle(10000)  # 把10000内全部打散"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 数据与处理\n",
    "def preprocess(x,y):\n",
    "    x = tf.cast(x,dtype=tf.float32)/255.  # 转32位,/255缩小范围\n",
    "    y = tf.cast(y,dtype=tf.int32)\n",
    "    y = tf.one_hot(y,depth=10)\n",
    "    return x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "db2 = db.map(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(TensorShape([32, 32, 3]), TensorShape([1, 10]))"
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "res = next(iter(db2))\n",
    "res[0].shape,res[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "(TensorShape([32, 32, 32, 3]), TensorShape([32, 1, 10]))"
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "# batch \n",
    "db3 = db2.batch(32)\n",
    "res = next(iter(db3))\n",
    "res[0].shape,res[1].shape"
   ]
  }
 ]
}